{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fcb7879d-f417-47e2-b723-c1e368d19873",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -q \"torch==2.1.2\" tensorboard wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e47209a1-97a0-4151-8344-d6f109eb6287",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Install Pytorch & other libraries\n",
    "!pip install -q \"torch==2.1.2\" tensorboard\n",
    "\n",
    "# Install Hugging Face libraries\n",
    "!pip install  -q --upgrade \\\n",
    "  \"transformers==4.36.2\" \\\n",
    "  \"datasets==2.16.1\" \\\n",
    "  \"accelerate==0.26.1\" \\\n",
    "  \"evaluate==0.4.1\" \\\n",
    "  \"bitsandbytes==0.42.0\" \\\n",
    "  \"trl==0.7.10\"  \\\n",
    "  \"peft==0.7.1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "88054d2a-4b6e-40bf-b44a-f501fe881009",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting flash-attn\n",
      "  Downloading flash_attn-2.5.0.tar.gz (2.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (from flash-attn) (2.1.2)\n",
      "Collecting einops\n",
      "  Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from flash-attn) (23.0)\n",
      "Collecting ninja\n",
      "  Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl (307 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.2/307.2 kB\u001b[0m \u001b[31m31.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (11.0.2.54)\n",
      "Requirement already satisfied: triton==2.1.0 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (2.1.0)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (2.18.1)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (8.9.2.26)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (3.1)\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (2023.10.0)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (12.1.0.106)\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (4.5.0)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (11.4.5.107)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (1.12)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch->flash-attn) (3.9.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /opt/conda/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->flash-attn) (12.3.101)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch->flash-attn) (2.1.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch->flash-attn) (1.3.0)\n",
      "Building wheels for collected packages: flash-attn\n",
      "  Building wheel for flash-attn (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for flash-attn: filename=flash_attn-2.5.0-cp310-cp310-linux_x86_64.whl size=120823033 sha256=3335e74258645eb190597754d42c2fee391fbdeb772847f9e1de12da60450a33\n",
      "  Stored in directory: /root/.cache/pip/wheels/9e/c3/22/a576eb5627fb2c30dc4679a33d67d34d922d6dbeb24a9119b2\n",
      "Successfully built flash-attn\n",
      "Installing collected packages: ninja, einops, flash-attn\n",
      "Successfully installed einops-0.7.0 flash-attn-2.5.0 ninja-1.11.1.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install flash-attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a45add05-1c1f-4398-9211-6130564fdb11",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git config --global credential.helper store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "993f8ad3-185b-4f58-a85c-f1523b0f5e53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token is valid (permission: write).\n",
      "Your token has been saved in your configured git credential helpers (store).\n",
      "Your token has been saved to /root/.cache/huggingface/token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import login\n",
    "\n",
    "login(\n",
    "  token=\"\", # ADD YOUR TOKEN HERE\n",
    "  add_to_git_credential=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1666f81b-5f44-422a-aa32-64c1d0ae986d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14694553f76f4e55b8fd7ecbdbafb8e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/12500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'content': 'दक्षिण पूर्व एशिया में यात्रा के अनुभवों के बारे में ब्लॉग पोस्ट खोजें।', 'role': 'system'}, {'content': 'मैंने हाल ही में दक्षिण पूर्व एशिया की यात्रा की और बैंकॉक में जीवंत स्ट्रीट फूड दृश्य की खोज करने, हनोई के समृद्ध इतिहास में खुद को डुबोने और बाली में छिपे हुए रत्नों की खोज करने में एक अद्भुत समय बिताया। अंदरूनी सुझावों और सिफारिशों के लिए मेरे ब्लॉग पोस्ट को देखें!', 'role': 'user'}, {'content': 'अद्वितीय आराम और लुभावने दृश्यों की पेशकश करते हुए दक्षिण पूर्व एशिया में शीर्ष 10 लक्जरी रिसॉर्ट्स की खोज करें। इन विशिष्ट गंतव्यों में विश्व स्तरीय सुविधाओं और लाड़-प्यार की सेवाओं में शामिल हों, जो इस क्षेत्र में एक आरामदायक सैर के लिए एकदम सही हैं।', 'role': 'assistant'}]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb7e76c8676240798d4477ddb4c78524",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/10 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "834ad4f51e994183935e13eb67a701cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "12038569"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "def create_conversation(sample):\n",
    "  return {\n",
    "    \"messages\": [\n",
    "      {\"role\": \"system\", \"content\": sample[\"task\"]},\n",
    "      {\"role\": \"user\", \"content\": sample[\"query\"]},\n",
    "      {\"role\": \"assistant\", \"content\": sample[\"pos\"]}\n",
    "    ]\n",
    "  }\n",
    "\n",
    "# Load dataset from the hub\n",
    "dataset = load_dataset(\"TokenBender/sentence_retrieval_hindi_SFT\", split=\"train\")\n",
    "dataset = dataset.shuffle().select(range(12500))\n",
    "\n",
    "# Convert dataset to OAI messages\n",
    "dataset = dataset.map(create_conversation, remove_columns=dataset.features,batched=False)\n",
    "# split dataset into 10,000 training samples and 2,500 test samples\n",
    "dataset = dataset.train_test_split(test_size=2500/12500)\n",
    "\n",
    "print(dataset[\"train\"][345][\"messages\"])\n",
    "\n",
    "# save datasets to disk\n",
    "dataset[\"train\"].to_json(\"train_dataset.json\", orient=\"records\")\n",
    "dataset[\"test\"].to_json(\"test_dataset.json\", orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1f5afb85-c24a-40b4-8508-f705680a92d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "327dcbfc8c5e43eab71b6621603c2add",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# Load jsonl data from disk\n",
    "dataset = load_dataset(\"json\", data_files=\"train_dataset.json\", split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8f6cb2cf-6e8b-4351-aa2e-d254a0ab7290",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mahm-rimer\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: WANDB_PROJECT=hindi_sft_test_tinyllama\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
    "from trl import setup_chat_format\n",
    "import wandb\n",
    "wandb.login()\n",
    "%env WANDB_PROJECT=hindi_sft_test_tinyllama\n",
    "\n",
    "# Hugging Face model id\n",
    "model_id = \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\" # or `mistralai/Mistral-7B-v0.1`\n",
    "\n",
    "# BitsAndBytesConfig int-4 config\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type=\"nf4\", bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "# Load model and tokenizer\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    device_map=\"auto\",\n",
    "    attn_implementation=\"flash_attention_2\",\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    quantization_config=bnb_config\n",
    ")\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "tokenizer.padding_side = 'right' # to prevent warnings\n",
    "\n",
    "# # set chat template to OAI chatML, remove if you start from a fine-tuned model\n",
    "model, tokenizer = setup_chat_format(model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "47ddb2f5-46c2-4be8-8b98-cf19a227a03c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import LoraConfig\n",
    "\n",
    "# LoRA config based on QLoRA paper & Sebastian Raschka experiment\n",
    "peft_config = LoraConfig(\n",
    "        lora_alpha=16,\n",
    "        lora_dropout=0.05,\n",
    "        r=32,\n",
    "        bias=\"none\",\n",
    "        target_modules=['k_proj', 'gate_proj', 'v_proj', 'up_proj', 'q_proj', 'o_proj', 'down_proj'],\n",
    "        task_type=\"CAUSAL_LM\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2cd0378d-6f5b-4b5e-bad9-ba36512d91e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "args = TrainingArguments(\n",
    "    output_dir=\"tinyllama_hindi_sentence_retrieval_sft\", # directory to save and repository id\n",
    "    num_train_epochs=1,                     # number of training epochs\n",
    "    per_device_train_batch_size=4,          # batch size per device during training\n",
    "    gradient_accumulation_steps=2,          # number of steps before performing a backward/update pass\n",
    "    gradient_checkpointing=True,            # use gradient checkpointing to save memory\n",
    "    optim=\"adamw_torch_fused\",              # use fused adamw optimizer\n",
    "    logging_steps=1,                       # log every 10 steps\n",
    "    save_steps=0.3,                  # save checkpoint every epoch\n",
    "    learning_rate=2e-4,                     # learning rate, based on QLoRA paper\n",
    "    bf16=True,                              # use bfloat16 precision\n",
    "    tf32=True,                              # use tf32 precision\n",
    "    max_grad_norm=0.3,                      # max gradient norm based on QLoRA paper\n",
    "    warmup_ratio=0.03,                      # warmup ratio based on QLoRA paper\n",
    "    lr_scheduler_type=\"constant\",           # use constant learning rate scheduler\n",
    "    push_to_hub=True,                       # push model to hub\n",
    "    report_to=\"wandb\",                      # report metrics to wandb\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1952b73a-7d6e-404b-8cc5-38c75e463af1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "653c699b78e64a0ba64ecffb0053d128",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from trl import SFTTrainer\n",
    "\n",
    "max_seq_length = 2048 # max sequence length for model and packing of the dataset\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=dataset,\n",
    "    peft_config=peft_config,\n",
    "    max_seq_length=max_seq_length,\n",
    "    tokenizer=tokenizer,\n",
    "    packing=True,\n",
    "    dataset_kwargs={\n",
    "        \"add_special_tokens\": False,  # We template with special tokens\n",
    "        \"append_concat_token\": False, # No need to add additional separator token\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bdcc866c-b520-445e-bbb7-d6d70a3c12cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href='https://wandb.me/wandb-init' target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspace/wandb/run-20240128_191736-13n2gxgh</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/ahm-rimer/hindi_sft_test_tinyllama/runs/13n2gxgh' target=\"_blank\">twilight-plant-1</a></strong> to <a href='https://wandb.ai/ahm-rimer/hindi_sft_test_tinyllama' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/ahm-rimer/hindi_sft_test_tinyllama' target=\"_blank\">https://wandb.ai/ahm-rimer/hindi_sft_test_tinyllama</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/ahm-rimer/hindi_sft_test_tinyllama/runs/13n2gxgh' target=\"_blank\">https://wandb.ai/ahm-rimer/hindi_sft_test_tinyllama/runs/13n2gxgh</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n",
      "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in torch.bfloat16.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='622' max='622' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [622/622 41:28, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.197200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.141000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.131400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.086400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.089900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.004200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>1.032800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>1.062700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1.045000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.994600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.979000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.966600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.980000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.914500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.952300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.915400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.941800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.949200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.864800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.937400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.959400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.929800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.892400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.900700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.891200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.850800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.912600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.832900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.846400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.840500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.856000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.793800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.901100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.871500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.834300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.832300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.810800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.840100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.886200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.823800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.823300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.868200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.851900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.845500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.829100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.826400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.850900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.808600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.832700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>0.784200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>0.810200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>0.785500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>0.776400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>0.784800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>0.796800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>0.803300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>0.776000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>0.829500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.748200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61</td>\n",
       "      <td>0.778100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62</td>\n",
       "      <td>0.757000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63</td>\n",
       "      <td>0.818700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64</td>\n",
       "      <td>0.846200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65</td>\n",
       "      <td>0.811500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66</td>\n",
       "      <td>0.804400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67</td>\n",
       "      <td>0.752500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68</td>\n",
       "      <td>0.768000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69</td>\n",
       "      <td>0.773200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>0.763800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>0.725100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72</td>\n",
       "      <td>0.794800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>0.734700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>0.732800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>0.758000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76</td>\n",
       "      <td>0.710200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77</td>\n",
       "      <td>0.781100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78</td>\n",
       "      <td>0.753400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>79</td>\n",
       "      <td>0.701600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>0.758800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81</td>\n",
       "      <td>0.837000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>82</td>\n",
       "      <td>0.789900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>83</td>\n",
       "      <td>0.775300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>84</td>\n",
       "      <td>0.737000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85</td>\n",
       "      <td>0.776300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>86</td>\n",
       "      <td>0.755400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>87</td>\n",
       "      <td>0.745100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>88</td>\n",
       "      <td>0.743800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>89</td>\n",
       "      <td>0.693900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>0.733400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>91</td>\n",
       "      <td>0.786900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>92</td>\n",
       "      <td>0.766600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>93</td>\n",
       "      <td>0.769400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>94</td>\n",
       "      <td>0.720600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95</td>\n",
       "      <td>0.730200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>0.729800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>97</td>\n",
       "      <td>0.740800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>0.767000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>0.757500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.737800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>101</td>\n",
       "      <td>0.728100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>102</td>\n",
       "      <td>0.755200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>103</td>\n",
       "      <td>0.698300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>104</td>\n",
       "      <td>0.711400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>105</td>\n",
       "      <td>0.766700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>106</td>\n",
       "      <td>0.749500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>107</td>\n",
       "      <td>0.705200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>108</td>\n",
       "      <td>0.680300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>109</td>\n",
       "      <td>0.674500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>0.706600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>111</td>\n",
       "      <td>0.759000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>112</td>\n",
       "      <td>0.699500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>113</td>\n",
       "      <td>0.709700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>114</td>\n",
       "      <td>0.714800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>115</td>\n",
       "      <td>0.708000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>116</td>\n",
       "      <td>0.700300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>117</td>\n",
       "      <td>0.673500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>118</td>\n",
       "      <td>0.760100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>119</td>\n",
       "      <td>0.694300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>0.706500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>121</td>\n",
       "      <td>0.721300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>122</td>\n",
       "      <td>0.698400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>123</td>\n",
       "      <td>0.738900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>124</td>\n",
       "      <td>0.729600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>125</td>\n",
       "      <td>0.696200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>126</td>\n",
       "      <td>0.676000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>127</td>\n",
       "      <td>0.695700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>128</td>\n",
       "      <td>0.729200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>129</td>\n",
       "      <td>0.730000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>0.719900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>131</td>\n",
       "      <td>0.726200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>132</td>\n",
       "      <td>0.693100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>133</td>\n",
       "      <td>0.706900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>134</td>\n",
       "      <td>0.708700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>135</td>\n",
       "      <td>0.691700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>136</td>\n",
       "      <td>0.682500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>137</td>\n",
       "      <td>0.727800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>138</td>\n",
       "      <td>0.633700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>139</td>\n",
       "      <td>0.710700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>0.653100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>141</td>\n",
       "      <td>0.717000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>142</td>\n",
       "      <td>0.732800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>143</td>\n",
       "      <td>0.677000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>144</td>\n",
       "      <td>0.688600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>145</td>\n",
       "      <td>0.673100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>146</td>\n",
       "      <td>0.678900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>147</td>\n",
       "      <td>0.679900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>148</td>\n",
       "      <td>0.667800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>149</td>\n",
       "      <td>0.643900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.679000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>151</td>\n",
       "      <td>0.666700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>152</td>\n",
       "      <td>0.695600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>153</td>\n",
       "      <td>0.655300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>154</td>\n",
       "      <td>0.710500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>155</td>\n",
       "      <td>0.659700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>156</td>\n",
       "      <td>0.717600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>157</td>\n",
       "      <td>0.657500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>158</td>\n",
       "      <td>0.657900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159</td>\n",
       "      <td>0.695600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>0.673400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>161</td>\n",
       "      <td>0.642500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>162</td>\n",
       "      <td>0.702800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>163</td>\n",
       "      <td>0.713500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>164</td>\n",
       "      <td>0.674100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>165</td>\n",
       "      <td>0.746000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>166</td>\n",
       "      <td>0.676800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>167</td>\n",
       "      <td>0.669100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>168</td>\n",
       "      <td>0.668800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>169</td>\n",
       "      <td>0.655000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170</td>\n",
       "      <td>0.684400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>171</td>\n",
       "      <td>0.688200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>172</td>\n",
       "      <td>0.705100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>173</td>\n",
       "      <td>0.669600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>174</td>\n",
       "      <td>0.654800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>175</td>\n",
       "      <td>0.691300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>176</td>\n",
       "      <td>0.640200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>177</td>\n",
       "      <td>0.691600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>178</td>\n",
       "      <td>0.701600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>179</td>\n",
       "      <td>0.718500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>0.629500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>181</td>\n",
       "      <td>0.706600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>182</td>\n",
       "      <td>0.661800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>183</td>\n",
       "      <td>0.649300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>184</td>\n",
       "      <td>0.687800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>185</td>\n",
       "      <td>0.623300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>186</td>\n",
       "      <td>0.729500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>187</td>\n",
       "      <td>0.645000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>188</td>\n",
       "      <td>0.723100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>189</td>\n",
       "      <td>0.665900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>190</td>\n",
       "      <td>0.628100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>191</td>\n",
       "      <td>0.707700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>192</td>\n",
       "      <td>0.676500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>193</td>\n",
       "      <td>0.644600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>194</td>\n",
       "      <td>0.658400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>195</td>\n",
       "      <td>0.729700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>196</td>\n",
       "      <td>0.668800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>197</td>\n",
       "      <td>0.672800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>198</td>\n",
       "      <td>0.667000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>199</td>\n",
       "      <td>0.679100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.656400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>201</td>\n",
       "      <td>0.633200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>202</td>\n",
       "      <td>0.651700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>203</td>\n",
       "      <td>0.648600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>204</td>\n",
       "      <td>0.603300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>205</td>\n",
       "      <td>0.655100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>206</td>\n",
       "      <td>0.637800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>207</td>\n",
       "      <td>0.624800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>208</td>\n",
       "      <td>0.635600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>209</td>\n",
       "      <td>0.640000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>210</td>\n",
       "      <td>0.693500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>211</td>\n",
       "      <td>0.677000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>212</td>\n",
       "      <td>0.625200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>213</td>\n",
       "      <td>0.668800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>214</td>\n",
       "      <td>0.633200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>215</td>\n",
       "      <td>0.643800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>216</td>\n",
       "      <td>0.677900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>217</td>\n",
       "      <td>0.602000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>218</td>\n",
       "      <td>0.616500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>219</td>\n",
       "      <td>0.653500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>0.641100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>221</td>\n",
       "      <td>0.624500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>222</td>\n",
       "      <td>0.684600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>223</td>\n",
       "      <td>0.670300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>224</td>\n",
       "      <td>0.675900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>225</td>\n",
       "      <td>0.609500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>226</td>\n",
       "      <td>0.600900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>227</td>\n",
       "      <td>0.642300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>228</td>\n",
       "      <td>0.607700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>229</td>\n",
       "      <td>0.666700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>230</td>\n",
       "      <td>0.613300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>231</td>\n",
       "      <td>0.661400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>232</td>\n",
       "      <td>0.661800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>233</td>\n",
       "      <td>0.627900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>234</td>\n",
       "      <td>0.707200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>235</td>\n",
       "      <td>0.611800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>236</td>\n",
       "      <td>0.611900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>237</td>\n",
       "      <td>0.574400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>238</td>\n",
       "      <td>0.623300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>239</td>\n",
       "      <td>0.681000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>0.622300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>241</td>\n",
       "      <td>0.651900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>242</td>\n",
       "      <td>0.614700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>243</td>\n",
       "      <td>0.654900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>244</td>\n",
       "      <td>0.663600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>245</td>\n",
       "      <td>0.670500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>246</td>\n",
       "      <td>0.619700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>247</td>\n",
       "      <td>0.586900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>248</td>\n",
       "      <td>0.644200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>249</td>\n",
       "      <td>0.614600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>0.641000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>251</td>\n",
       "      <td>0.633500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>252</td>\n",
       "      <td>0.645700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>253</td>\n",
       "      <td>0.672500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>254</td>\n",
       "      <td>0.635300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>255</td>\n",
       "      <td>0.644100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>256</td>\n",
       "      <td>0.641300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>257</td>\n",
       "      <td>0.569300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>258</td>\n",
       "      <td>0.674100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>259</td>\n",
       "      <td>0.622000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>0.659600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>261</td>\n",
       "      <td>0.605200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>262</td>\n",
       "      <td>0.628800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>263</td>\n",
       "      <td>0.606600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>264</td>\n",
       "      <td>0.591900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>265</td>\n",
       "      <td>0.623100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>266</td>\n",
       "      <td>0.604400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>267</td>\n",
       "      <td>0.605600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>268</td>\n",
       "      <td>0.655400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>269</td>\n",
       "      <td>0.695500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>270</td>\n",
       "      <td>0.618400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>271</td>\n",
       "      <td>0.669500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>272</td>\n",
       "      <td>0.641000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>273</td>\n",
       "      <td>0.626000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>274</td>\n",
       "      <td>0.617500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>275</td>\n",
       "      <td>0.620000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>276</td>\n",
       "      <td>0.638700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>277</td>\n",
       "      <td>0.592700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>278</td>\n",
       "      <td>0.648200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>279</td>\n",
       "      <td>0.636100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>0.581300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>281</td>\n",
       "      <td>0.557300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>282</td>\n",
       "      <td>0.643300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>283</td>\n",
       "      <td>0.646800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>284</td>\n",
       "      <td>0.625300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>285</td>\n",
       "      <td>0.654400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>286</td>\n",
       "      <td>0.607100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>287</td>\n",
       "      <td>0.593400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>288</td>\n",
       "      <td>0.596900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>289</td>\n",
       "      <td>0.539600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>290</td>\n",
       "      <td>0.620200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>291</td>\n",
       "      <td>0.595400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>292</td>\n",
       "      <td>0.589700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>293</td>\n",
       "      <td>0.642000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>294</td>\n",
       "      <td>0.569100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>295</td>\n",
       "      <td>0.595600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>296</td>\n",
       "      <td>0.594500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297</td>\n",
       "      <td>0.646400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>298</td>\n",
       "      <td>0.630300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299</td>\n",
       "      <td>0.658800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.614100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>301</td>\n",
       "      <td>0.663500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>302</td>\n",
       "      <td>0.649000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>303</td>\n",
       "      <td>0.609400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>304</td>\n",
       "      <td>0.615200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>305</td>\n",
       "      <td>0.628400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>306</td>\n",
       "      <td>0.599600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>307</td>\n",
       "      <td>0.611500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>308</td>\n",
       "      <td>0.605600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>309</td>\n",
       "      <td>0.590200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>310</td>\n",
       "      <td>0.607900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>311</td>\n",
       "      <td>0.627600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>312</td>\n",
       "      <td>0.623900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>313</td>\n",
       "      <td>0.643100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>314</td>\n",
       "      <td>0.609400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>315</td>\n",
       "      <td>0.582000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>316</td>\n",
       "      <td>0.574000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>317</td>\n",
       "      <td>0.600700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>318</td>\n",
       "      <td>0.599200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>319</td>\n",
       "      <td>0.596700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>320</td>\n",
       "      <td>0.620400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>321</td>\n",
       "      <td>0.579700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>322</td>\n",
       "      <td>0.666400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>323</td>\n",
       "      <td>0.576000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>324</td>\n",
       "      <td>0.644500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>325</td>\n",
       "      <td>0.593400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>326</td>\n",
       "      <td>0.624900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>327</td>\n",
       "      <td>0.577800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>328</td>\n",
       "      <td>0.618400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>329</td>\n",
       "      <td>0.586700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>330</td>\n",
       "      <td>0.608200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>331</td>\n",
       "      <td>0.598000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>332</td>\n",
       "      <td>0.580400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>333</td>\n",
       "      <td>0.624300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>334</td>\n",
       "      <td>0.567800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>335</td>\n",
       "      <td>0.593700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>336</td>\n",
       "      <td>0.554100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>337</td>\n",
       "      <td>0.719700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>338</td>\n",
       "      <td>0.551600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>339</td>\n",
       "      <td>0.565500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>340</td>\n",
       "      <td>0.590000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>341</td>\n",
       "      <td>0.591700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>342</td>\n",
       "      <td>0.584800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>343</td>\n",
       "      <td>0.605800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>344</td>\n",
       "      <td>0.641100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>345</td>\n",
       "      <td>0.588000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>346</td>\n",
       "      <td>0.615200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>347</td>\n",
       "      <td>0.567100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>348</td>\n",
       "      <td>0.610200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>349</td>\n",
       "      <td>0.626000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>0.610900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>351</td>\n",
       "      <td>0.591800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>352</td>\n",
       "      <td>0.585600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>353</td>\n",
       "      <td>0.599700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>354</td>\n",
       "      <td>0.606800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>355</td>\n",
       "      <td>0.571400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>356</td>\n",
       "      <td>0.612700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>357</td>\n",
       "      <td>0.585900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>358</td>\n",
       "      <td>0.625800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>359</td>\n",
       "      <td>0.642900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>360</td>\n",
       "      <td>0.550300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>361</td>\n",
       "      <td>0.566100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>362</td>\n",
       "      <td>0.604000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>363</td>\n",
       "      <td>0.600600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>364</td>\n",
       "      <td>0.627300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>365</td>\n",
       "      <td>0.521300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>366</td>\n",
       "      <td>0.622500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>367</td>\n",
       "      <td>0.562700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>368</td>\n",
       "      <td>0.577400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>369</td>\n",
       "      <td>0.546600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>370</td>\n",
       "      <td>0.576200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>371</td>\n",
       "      <td>0.582100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>372</td>\n",
       "      <td>0.604100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>373</td>\n",
       "      <td>0.632300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>374</td>\n",
       "      <td>0.626800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>375</td>\n",
       "      <td>0.593400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>376</td>\n",
       "      <td>0.614400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>377</td>\n",
       "      <td>0.566200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>378</td>\n",
       "      <td>0.608800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>379</td>\n",
       "      <td>0.562100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>380</td>\n",
       "      <td>0.564600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>381</td>\n",
       "      <td>0.576500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>382</td>\n",
       "      <td>0.572100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>383</td>\n",
       "      <td>0.573600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>384</td>\n",
       "      <td>0.600700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>385</td>\n",
       "      <td>0.500700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>386</td>\n",
       "      <td>0.618800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>387</td>\n",
       "      <td>0.561100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>388</td>\n",
       "      <td>0.605900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>389</td>\n",
       "      <td>0.579300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>390</td>\n",
       "      <td>0.615000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>391</td>\n",
       "      <td>0.540200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>392</td>\n",
       "      <td>0.561600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>393</td>\n",
       "      <td>0.563700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>394</td>\n",
       "      <td>0.573000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>395</td>\n",
       "      <td>0.597400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>396</td>\n",
       "      <td>0.554300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>397</td>\n",
       "      <td>0.565700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>398</td>\n",
       "      <td>0.620500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>399</td>\n",
       "      <td>0.513900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.539300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>401</td>\n",
       "      <td>0.609100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>402</td>\n",
       "      <td>0.547700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>403</td>\n",
       "      <td>0.557300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>404</td>\n",
       "      <td>0.585300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>405</td>\n",
       "      <td>0.586300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>406</td>\n",
       "      <td>0.598300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>407</td>\n",
       "      <td>0.547800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>408</td>\n",
       "      <td>0.530200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>409</td>\n",
       "      <td>0.620100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>410</td>\n",
       "      <td>0.568500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>411</td>\n",
       "      <td>0.596900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>412</td>\n",
       "      <td>0.610400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>413</td>\n",
       "      <td>0.587900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>414</td>\n",
       "      <td>0.553600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>415</td>\n",
       "      <td>0.608500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>416</td>\n",
       "      <td>0.519700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>417</td>\n",
       "      <td>0.613200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>418</td>\n",
       "      <td>0.579200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>419</td>\n",
       "      <td>0.613900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>420</td>\n",
       "      <td>0.596300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>421</td>\n",
       "      <td>0.546900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>422</td>\n",
       "      <td>0.589300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>423</td>\n",
       "      <td>0.589900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>424</td>\n",
       "      <td>0.580600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>425</td>\n",
       "      <td>0.584400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>426</td>\n",
       "      <td>0.639800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>427</td>\n",
       "      <td>0.584700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>428</td>\n",
       "      <td>0.596400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>429</td>\n",
       "      <td>0.532800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>430</td>\n",
       "      <td>0.629400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>431</td>\n",
       "      <td>0.560600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>432</td>\n",
       "      <td>0.565700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>433</td>\n",
       "      <td>0.570000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>434</td>\n",
       "      <td>0.595200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>435</td>\n",
       "      <td>0.554300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>436</td>\n",
       "      <td>0.626400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>437</td>\n",
       "      <td>0.611700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>438</td>\n",
       "      <td>0.584300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>439</td>\n",
       "      <td>0.574700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>440</td>\n",
       "      <td>0.611400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>441</td>\n",
       "      <td>0.554900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>442</td>\n",
       "      <td>0.586000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>443</td>\n",
       "      <td>0.594200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>444</td>\n",
       "      <td>0.532100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>445</td>\n",
       "      <td>0.580600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>446</td>\n",
       "      <td>0.590500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>447</td>\n",
       "      <td>0.551300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>448</td>\n",
       "      <td>0.556200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>449</td>\n",
       "      <td>0.566300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>0.600100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>451</td>\n",
       "      <td>0.597400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>452</td>\n",
       "      <td>0.526500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>453</td>\n",
       "      <td>0.609900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>454</td>\n",
       "      <td>0.572600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>455</td>\n",
       "      <td>0.629700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>456</td>\n",
       "      <td>0.509900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>457</td>\n",
       "      <td>0.585800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>458</td>\n",
       "      <td>0.569600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>459</td>\n",
       "      <td>0.541300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>460</td>\n",
       "      <td>0.525000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>461</td>\n",
       "      <td>0.543200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>462</td>\n",
       "      <td>0.597100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>463</td>\n",
       "      <td>0.539400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>464</td>\n",
       "      <td>0.566400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>465</td>\n",
       "      <td>0.594900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>466</td>\n",
       "      <td>0.595700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>467</td>\n",
       "      <td>0.530100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>468</td>\n",
       "      <td>0.525500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>469</td>\n",
       "      <td>0.540600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>470</td>\n",
       "      <td>0.577400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>471</td>\n",
       "      <td>0.543700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>472</td>\n",
       "      <td>0.534800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>473</td>\n",
       "      <td>0.607000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>474</td>\n",
       "      <td>0.624600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>475</td>\n",
       "      <td>0.571200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>476</td>\n",
       "      <td>0.500100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>477</td>\n",
       "      <td>0.571600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>478</td>\n",
       "      <td>0.548500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>479</td>\n",
       "      <td>0.546200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>480</td>\n",
       "      <td>0.550800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>481</td>\n",
       "      <td>0.553000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>482</td>\n",
       "      <td>0.541900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>483</td>\n",
       "      <td>0.520500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>484</td>\n",
       "      <td>0.566200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>485</td>\n",
       "      <td>0.573500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>486</td>\n",
       "      <td>0.581800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>487</td>\n",
       "      <td>0.622700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>488</td>\n",
       "      <td>0.547400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>489</td>\n",
       "      <td>0.566500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>490</td>\n",
       "      <td>0.542000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>491</td>\n",
       "      <td>0.544900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>492</td>\n",
       "      <td>0.541100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>493</td>\n",
       "      <td>0.515500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>494</td>\n",
       "      <td>0.587000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>495</td>\n",
       "      <td>0.518900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>496</td>\n",
       "      <td>0.514400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>497</td>\n",
       "      <td>0.545600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>498</td>\n",
       "      <td>0.595700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>499</td>\n",
       "      <td>0.551900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.539100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>501</td>\n",
       "      <td>0.548600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>502</td>\n",
       "      <td>0.556300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>503</td>\n",
       "      <td>0.523200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>504</td>\n",
       "      <td>0.556300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>505</td>\n",
       "      <td>0.558400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>506</td>\n",
       "      <td>0.508500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>507</td>\n",
       "      <td>0.553200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>508</td>\n",
       "      <td>0.557600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>509</td>\n",
       "      <td>0.572900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>510</td>\n",
       "      <td>0.597800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>511</td>\n",
       "      <td>0.524900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>512</td>\n",
       "      <td>0.529500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>513</td>\n",
       "      <td>0.566900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>514</td>\n",
       "      <td>0.562600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>515</td>\n",
       "      <td>0.546500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>516</td>\n",
       "      <td>0.517900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>517</td>\n",
       "      <td>0.531000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>518</td>\n",
       "      <td>0.571500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>519</td>\n",
       "      <td>0.503300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>520</td>\n",
       "      <td>0.578200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>521</td>\n",
       "      <td>0.598000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>522</td>\n",
       "      <td>0.505400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>523</td>\n",
       "      <td>0.533900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>524</td>\n",
       "      <td>0.527300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>525</td>\n",
       "      <td>0.552600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>526</td>\n",
       "      <td>0.554500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>527</td>\n",
       "      <td>0.534700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>528</td>\n",
       "      <td>0.561500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>529</td>\n",
       "      <td>0.553300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>530</td>\n",
       "      <td>0.509700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>531</td>\n",
       "      <td>0.531900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>532</td>\n",
       "      <td>0.525000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>533</td>\n",
       "      <td>0.571200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>534</td>\n",
       "      <td>0.525800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>535</td>\n",
       "      <td>0.593100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>536</td>\n",
       "      <td>0.545800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>537</td>\n",
       "      <td>0.522400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>538</td>\n",
       "      <td>0.588000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>539</td>\n",
       "      <td>0.556900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>540</td>\n",
       "      <td>0.553500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>541</td>\n",
       "      <td>0.561000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>542</td>\n",
       "      <td>0.546200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>543</td>\n",
       "      <td>0.510300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>544</td>\n",
       "      <td>0.552300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>545</td>\n",
       "      <td>0.526000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>546</td>\n",
       "      <td>0.531100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>547</td>\n",
       "      <td>0.509700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>548</td>\n",
       "      <td>0.482200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>549</td>\n",
       "      <td>0.547000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>0.532000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>551</td>\n",
       "      <td>0.534600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>552</td>\n",
       "      <td>0.546000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>553</td>\n",
       "      <td>0.542100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>554</td>\n",
       "      <td>0.518800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>555</td>\n",
       "      <td>0.603500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>556</td>\n",
       "      <td>0.514000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>557</td>\n",
       "      <td>0.538500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>558</td>\n",
       "      <td>0.551000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>559</td>\n",
       "      <td>0.548400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>560</td>\n",
       "      <td>0.542600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>561</td>\n",
       "      <td>0.533900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>562</td>\n",
       "      <td>0.572400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>563</td>\n",
       "      <td>0.556300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>564</td>\n",
       "      <td>0.538900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>565</td>\n",
       "      <td>0.586900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>566</td>\n",
       "      <td>0.518200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>567</td>\n",
       "      <td>0.472500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>568</td>\n",
       "      <td>0.554000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>569</td>\n",
       "      <td>0.530600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>570</td>\n",
       "      <td>0.552300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>571</td>\n",
       "      <td>0.523500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>572</td>\n",
       "      <td>0.586100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>573</td>\n",
       "      <td>0.540100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>574</td>\n",
       "      <td>0.561500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>575</td>\n",
       "      <td>0.540900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>576</td>\n",
       "      <td>0.525000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>577</td>\n",
       "      <td>0.542000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>578</td>\n",
       "      <td>0.605800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>579</td>\n",
       "      <td>0.549400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>580</td>\n",
       "      <td>0.508100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>581</td>\n",
       "      <td>0.523500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>582</td>\n",
       "      <td>0.526300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>583</td>\n",
       "      <td>0.521100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>584</td>\n",
       "      <td>0.525300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>585</td>\n",
       "      <td>0.523600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>586</td>\n",
       "      <td>0.506800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>587</td>\n",
       "      <td>0.547200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>588</td>\n",
       "      <td>0.550000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>589</td>\n",
       "      <td>0.571600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>590</td>\n",
       "      <td>0.539200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>591</td>\n",
       "      <td>0.561000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>592</td>\n",
       "      <td>0.529800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>593</td>\n",
       "      <td>0.488400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>594</td>\n",
       "      <td>0.512300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>595</td>\n",
       "      <td>0.503700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>596</td>\n",
       "      <td>0.520400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>597</td>\n",
       "      <td>0.523200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>598</td>\n",
       "      <td>0.527600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>599</td>\n",
       "      <td>0.569400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.515700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>601</td>\n",
       "      <td>0.540700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>602</td>\n",
       "      <td>0.504500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>603</td>\n",
       "      <td>0.523900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>604</td>\n",
       "      <td>0.527400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>605</td>\n",
       "      <td>0.539900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>606</td>\n",
       "      <td>0.507100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>607</td>\n",
       "      <td>0.484200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>608</td>\n",
       "      <td>0.525100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>609</td>\n",
       "      <td>0.568100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>610</td>\n",
       "      <td>0.565100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>611</td>\n",
       "      <td>0.535700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>612</td>\n",
       "      <td>0.507300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>613</td>\n",
       "      <td>0.529300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>614</td>\n",
       "      <td>0.543900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>615</td>\n",
       "      <td>0.531400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>616</td>\n",
       "      <td>0.520300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>617</td>\n",
       "      <td>0.527800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>618</td>\n",
       "      <td>0.560800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>619</td>\n",
       "      <td>0.522200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>620</td>\n",
       "      <td>0.491600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>621</td>\n",
       "      <td>0.548300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>622</td>\n",
       "      <td>0.560200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c3e01f84ff845d697a1bbe5abb1ba5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "adapter_model.safetensors:   0%|          | 0.00/50.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# start training, the model will be automatically saved to the hub and the output directory\n",
    "trainer.train()\n",
    "\n",
    "# save model\n",
    "trainer.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6bf54afa-aeee-4bdc-ae7d-aa11bc0d2d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# free the memory again\n",
    "del model\n",
    "del trainer\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9dceb05a-8231-4b27-b35e-a0357be65c79",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#### COMMENT IN TO MERGE PEFT AND BASE MODEL ####\n",
    "from peft import PeftModel, PeftConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import AutoPeftModelForCausalLM\n",
    "\n",
    "# # Load PEFT model on CPU\n",
    "config = PeftConfig.from_pretrained(args.output_dir)\n",
    "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,low_cpu_mem_usage=True)\n",
    "tokenizer = AutoTokenizer.from_pretrained(args.output_dir)\n",
    "model.resize_token_embeddings(len(tokenizer))\n",
    "model = PeftModel.from_pretrained(model, args.output_dir)\n",
    "model = AutoPeftModelForCausalLM.from_pretrained(\n",
    "    args.output_dir,\n",
    "    torch_dtype=torch.float16,\n",
    "    low_cpu_mem_usage=True,\n",
    ")\n",
    "# # Merge LoRA and base model and save\n",
    "merged_model = model.merge_and_unload()\n",
    "merged_model.save_pretrained(args.output_dir,safe_serialization=True, max_shard_size=\"2GB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "33a1999f-a6dc-42b3-8204-0a3f4103fdfd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The model 'PeftModelForCausalLM' is not supported for text-generation. Supported models are ['BartForCausalLM', 'BertLMHeadModel', 'BertGenerationDecoder', 'BigBirdForCausalLM', 'BigBirdPegasusForCausalLM', 'BioGptForCausalLM', 'BlenderbotForCausalLM', 'BlenderbotSmallForCausalLM', 'BloomForCausalLM', 'CamembertForCausalLM', 'LlamaForCausalLM', 'CodeGenForCausalLM', 'CpmAntForCausalLM', 'CTRLLMHeadModel', 'Data2VecTextForCausalLM', 'ElectraForCausalLM', 'ErnieForCausalLM', 'FalconForCausalLM', 'FuyuForCausalLM', 'GitForCausalLM', 'GPT2LMHeadModel', 'GPT2LMHeadModel', 'GPTBigCodeForCausalLM', 'GPTNeoForCausalLM', 'GPTNeoXForCausalLM', 'GPTNeoXJapaneseForCausalLM', 'GPTJForCausalLM', 'LlamaForCausalLM', 'MarianForCausalLM', 'MBartForCausalLM', 'MegaForCausalLM', 'MegatronBertForCausalLM', 'MistralForCausalLM', 'MixtralForCausalLM', 'MptForCausalLM', 'MusicgenForCausalLM', 'MvpForCausalLM', 'OpenLlamaForCausalLM', 'OpenAIGPTLMHeadModel', 'OPTForCausalLM', 'PegasusForCausalLM', 'PersimmonForCausalLM', 'PhiForCausalLM', 'PLBartForCausalLM', 'ProphetNetForCausalLM', 'QDQBertLMHeadModel', 'ReformerModelWithLMHead', 'RemBertForCausalLM', 'RobertaForCausalLM', 'RobertaPreLayerNormForCausalLM', 'RoCBertForCausalLM', 'RoFormerForCausalLM', 'RwkvForCausalLM', 'Speech2Text2ForCausalLM', 'TransfoXLLMHeadModel', 'TrOCRForCausalLM', 'WhisperForCausalLM', 'XGLMForCausalLM', 'XLMWithLMHeadModel', 'XLMProphetNetForCausalLM', 'XLMRobertaForCausalLM', 'XLMRobertaXLForCausalLM', 'XLNetLMHeadModel', 'XmodForCausalLM'].\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from peft import AutoPeftModelForCausalLM\n",
    "from transformers import AutoTokenizer, pipeline\n",
    "\n",
    "#peft_model_id = \"./tinyllama_hindi_sft_sentence_retrieval\"\n",
    "peft_model_id = args.output_dir\n",
    "\n",
    "# Load Model with PEFT adapter\n",
    "model = AutoPeftModelForCausalLM.from_pretrained(\n",
    "  peft_model_id,\n",
    "  device_map=\"auto\",\n",
    "  torch_dtype=torch.float16\n",
    ")\n",
    "# load into pipeline\n",
    "pipe = pipeline(\"text-generation\", model=model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0fc13edb-b8a4-40cd-b354-c4b8b36ff284",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4748f445714646138b2f55643478a4b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:389: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.1` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:394: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.1` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
      "  warnings.warn(\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [32,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [33,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [34,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [35,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [36,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [37,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [38,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [39,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [40,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [41,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [42,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [43,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [44,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [45,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [46,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [47,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [48,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [49,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [50,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [51,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [52,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [53,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [54,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [55,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [56,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [57,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [58,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [59,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [60,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [61,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [62,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:1292: indexSelectLargeIndex: block: [292,0,0], thread: [63,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[13], line 11\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;66;03m# Test on sample\u001b[39;00m\n\u001b[1;32m     10\u001b[0m prompt \u001b[38;5;241m=\u001b[39m pipe\u001b[38;5;241m.\u001b[39mtokenizer\u001b[38;5;241m.\u001b[39mapply_chat_template(eval_dataset[rand_idx][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m][:\u001b[38;5;241m2\u001b[39m], tokenize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, add_generation_prompt\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m---> 11\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mpipe\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_new_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdo_sample\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtop_k\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m50\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtop_p\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meos_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpipe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meos_token_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpad_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpipe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpad_token_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mQuery:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00meval_dataset[rand_idx][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;241m1\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOriginal Answer:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00meval_dataset[rand_idx][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;241m2\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/pipelines/text_generation.py:208\u001b[0m, in \u001b[0;36mTextGenerationPipeline.__call__\u001b[0;34m(self, text_inputs, **kwargs)\u001b[0m\n\u001b[1;32m    167\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, text_inputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    168\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    169\u001b[0m \u001b[38;5;124;03m    Complete the prompt(s) given as inputs.\u001b[39;00m\n\u001b[1;32m    170\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[38;5;124;03m          ids of the generated text.\u001b[39;00m\n\u001b[1;32m    207\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 208\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtext_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/pipelines/base.py:1140\u001b[0m, in \u001b[0;36mPipeline.__call__\u001b[0;34m(self, inputs, num_workers, batch_size, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1132\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mnext\u001b[39m(\n\u001b[1;32m   1133\u001b[0m         \u001b[38;5;28miter\u001b[39m(\n\u001b[1;32m   1134\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_iterator(\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1137\u001b[0m         )\n\u001b[1;32m   1138\u001b[0m     )\n\u001b[1;32m   1139\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1140\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_single\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpreprocess_params\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforward_params\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpostprocess_params\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/pipelines/base.py:1147\u001b[0m, in \u001b[0;36mPipeline.run_single\u001b[0;34m(self, inputs, preprocess_params, forward_params, postprocess_params)\u001b[0m\n\u001b[1;32m   1145\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrun_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, inputs, preprocess_params, forward_params, postprocess_params):\n\u001b[1;32m   1146\u001b[0m     model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpreprocess(inputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mpreprocess_params)\n\u001b[0;32m-> 1147\u001b[0m     model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mforward_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1148\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpostprocess(model_outputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mpostprocess_params)\n\u001b[1;32m   1149\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/pipelines/base.py:1046\u001b[0m, in \u001b[0;36mPipeline.forward\u001b[0;34m(self, model_inputs, **forward_params)\u001b[0m\n\u001b[1;32m   1044\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m inference_context():\n\u001b[1;32m   1045\u001b[0m         model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_ensure_tensor_on_device(model_inputs, device\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[0;32m-> 1046\u001b[0m         model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mforward_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1047\u001b[0m         model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_ensure_tensor_on_device(model_outputs, device\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[1;32m   1048\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/pipelines/text_generation.py:271\u001b[0m, in \u001b[0;36mTextGenerationPipeline._forward\u001b[0;34m(self, model_inputs, **generate_kwargs)\u001b[0m\n\u001b[1;32m    268\u001b[0m         generate_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmin_length\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m prefix_length\n\u001b[1;32m    270\u001b[0m \u001b[38;5;66;03m# BS x SL\u001b[39;00m\n\u001b[0;32m--> 271\u001b[0m generated_sequence \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mgenerate_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    272\u001b[0m out_b \u001b[38;5;241m=\u001b[39m generated_sequence\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    273\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mframework \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/peft/peft_model.py:1130\u001b[0m, in \u001b[0;36mPeftModelForCausalLM.generate\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m   1128\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_model\u001b[38;5;241m.\u001b[39mgeneration_config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgeneration_config\n\u001b[1;32m   1129\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1130\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbase_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1131\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m   1132\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_model\u001b[38;5;241m.\u001b[39mprepare_inputs_for_generation \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_model_prepare_inputs_for_generation\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m    113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    114\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py:1718\u001b[0m, in \u001b[0;36mGenerationMixin.generate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m   1701\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39massisted_decoding(\n\u001b[1;32m   1702\u001b[0m         input_ids,\n\u001b[1;32m   1703\u001b[0m         assistant_model\u001b[38;5;241m=\u001b[39massistant_model,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1714\u001b[0m         \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmodel_kwargs,\n\u001b[1;32m   1715\u001b[0m     )\n\u001b[1;32m   1716\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m generation_mode \u001b[38;5;241m==\u001b[39m GenerationMode\u001b[38;5;241m.\u001b[39mGREEDY_SEARCH:\n\u001b[1;32m   1717\u001b[0m     \u001b[38;5;66;03m# 11. run greedy search\u001b[39;00m\n\u001b[0;32m-> 1718\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgreedy_search\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1719\u001b[0m \u001b[43m        \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1720\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlogits_processor\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlogits_processor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1721\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstopping_criteria\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstopping_criteria\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1722\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpad_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpad_token_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1723\u001b[0m \u001b[43m        \u001b[49m\u001b[43meos_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meos_token_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1724\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_scores\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_scores\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1725\u001b[0m \u001b[43m        \u001b[49m\u001b[43mreturn_dict_in_generate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreturn_dict_in_generate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1726\u001b[0m \u001b[43m        \u001b[49m\u001b[43msynced_gpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msynced_gpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1727\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstreamer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstreamer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1728\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1729\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1731\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m generation_mode \u001b[38;5;241m==\u001b[39m GenerationMode\u001b[38;5;241m.\u001b[39mCONTRASTIVE_SEARCH:\n\u001b[1;32m   1732\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m model_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muse_cache\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py:2579\u001b[0m, in \u001b[0;36mGenerationMixin.greedy_search\u001b[0;34m(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)\u001b[0m\n\u001b[1;32m   2576\u001b[0m model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_inputs_for_generation(input_ids, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmodel_kwargs)\n\u001b[1;32m   2578\u001b[0m \u001b[38;5;66;03m# forward pass to get next token\u001b[39;00m\n\u001b[0;32m-> 2579\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2580\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2581\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   2582\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2583\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2584\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2586\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m synced_gpus \u001b[38;5;129;01mand\u001b[39;00m this_peer_finished:\n\u001b[1;32m   2587\u001b[0m     \u001b[38;5;28;01mcontinue\u001b[39;00m  \u001b[38;5;66;03m# don't waste resources running the code we don't need\u001b[39;00m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:1181\u001b[0m, in \u001b[0;36mLlamaForCausalLM.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1178\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[1;32m   1180\u001b[0m \u001b[38;5;66;03m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[39;00m\n\u001b[0;32m-> 1181\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1182\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1183\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1184\u001b[0m \u001b[43m    \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1185\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1186\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1187\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1188\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1189\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1190\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1191\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1193\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1194\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mpretraining_tp \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:1033\u001b[0m, in \u001b[0;36mLlamaModel.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1029\u001b[0m     attention_mask \u001b[38;5;241m=\u001b[39m attention_mask \u001b[38;5;28;01mif\u001b[39;00m (attention_mask \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01min\u001b[39;00m attention_mask) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1030\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_use_sdpa \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m output_attentions:\n\u001b[1;32m   1031\u001b[0m     \u001b[38;5;66;03m# output_attentions=True can not be supported when using SDPA, and we fall back on\u001b[39;00m\n\u001b[1;32m   1032\u001b[0m     \u001b[38;5;66;03m# the manual implementation that requires a 4D causal mask in all cases.\u001b[39;00m\n\u001b[0;32m-> 1033\u001b[0m     attention_mask \u001b[38;5;241m=\u001b[39m \u001b[43m_prepare_4d_causal_attention_mask_for_sdpa\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1034\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1035\u001b[0m \u001b[43m        \u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mseq_length\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1036\u001b[0m \u001b[43m        \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1037\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_values_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1038\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1039\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1040\u001b[0m     \u001b[38;5;66;03m# 4d mask is passed through the layers\u001b[39;00m\n\u001b[1;32m   1041\u001b[0m     attention_mask \u001b[38;5;241m=\u001b[39m _prepare_4d_causal_attention_mask(\n\u001b[1;32m   1042\u001b[0m         attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length\n\u001b[1;32m   1043\u001b[0m     )\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/modeling_attn_mask_utils.py:343\u001b[0m, in \u001b[0;36m_prepare_4d_causal_attention_mask_for_sdpa\u001b[0;34m(attention_mask, input_shape, inputs_embeds, past_key_values_length, sliding_window)\u001b[0m\n\u001b[1;32m    340\u001b[0m is_tracing \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mis_tracing()\n\u001b[1;32m    342\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attention_mask \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 343\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mall(attention_mask \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[1;32m    344\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m is_tracing:\n\u001b[1;32m    345\u001b[0m             \u001b[38;5;28;01mpass\u001b[39;00m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: device-side assert triggered\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from random import randint\n",
    "\n",
    "\n",
    "# Load our test dataset\n",
    "eval_dataset = load_dataset(\"json\", data_files=\"test_dataset.json\", split=\"train\")\n",
    "rand_idx = randint(0, len(eval_dataset))\n",
    "\n",
    "# Test on sample\n",
    "prompt = pipe.tokenizer.apply_chat_template(eval_dataset[rand_idx][\"messages\"][:2], tokenize=False, add_generation_prompt=True)\n",
    "outputs = pipe(prompt, max_new_tokens=256, do_sample=False, temperature=0.1, top_k=50, top_p=0.1, eos_token_id=pipe.tokenizer.eos_token_id, pad_token_id=pipe.tokenizer.pad_token_id)\n",
    "\n",
    "print(f\"Query:\\n{eval_dataset[rand_idx]['messages'][1]['content']}\")\n",
    "print(f\"Original Answer:\\n{eval_dataset[rand_idx]['messages'][2]['content']}\")\n",
    "print(f\"Generated Answer:\\n{outputs[0]['generated_text'][len(prompt):].strip()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d39baba5-8003-4451-b350-8fd7677edc30",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
